from typing import Any, Iterable
import scrapy
from scrapy.http import Response


# https://www.zenrows.com/blog/scraping-javascript-rendered-web-pages
class JsQuotesSpider(scrapy.Spider):
    name = "js_quotes"

    def start_requests(self) -> Iterable[scrapy.Request]:
        yield scrapy.Request(
            url="https://quotes.toscrape.com/js/",
            callback=self.parse,
            meta={'wait_selector': "small.author"}
        )

    def parse(self, response: Response, **kwargs: Any) -> Any:
        next_page = response.css("ul.pager li.next a")
        yield from response.follow_all(next_page, self.parse)

        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
                "tags": quote.css("div.tags a.tag::text").getall(),
            }
